from .utils import registry
from transformers import AutoTokenizer


@registry.tokenizers("hf.auto")
def build_auto_tokenizer(tokenizer_name_or_path: str) -> AutoTokenizer:
    """Build a tokenizer from a config."""
    return AutoTokenizer.from_pretrained(tokenizer_name_or_path)